import random
import time

from gxl_ai_utils.utils import utils_file

# data_config_path = "./tmp.yaml"
# # random.seed(10086)# 老的
# # 把当前时间戳作为随机种子
# # random.seed(int(time.time()))
# random.seed(7890)# 尝试一下新的顺序  #7890
# data_info_dict = utils_file.load_dict_from_yaml(data_config_path)
# if data_info_dict is None:
#     data_info_dict = {}
# total_list = []
# for data_info in data_info_dict.values():
#     data_path_i = data_info['path']
#     utils_file.logging_info(f'path:{data_path_i} ')
#     data_weight = int(float(data_info['weight']))
#     if data_weight == 0:
#         data_weight = float(data_info['weight'])
#         if data_weight >= 0:
#             utils_file.logging_info(f'data {data_path_i} weight is {data_weight}, will be used as a list')
#         final_data_list_i_tmp = utils_file.load_list_file_clean(data_path_i)
#         true_num = int(len(final_data_list_i_tmp)*data_weight)
#         final_data_list_i = utils_file.do_get_random_sublist(final_data_list_i_tmp, true_num)
#     else:
#         final_data_list_i = utils_file.load_list_file_clean(data_path_i) * data_weight
#     utils_file.logging_info(f'true load num is : {len(final_data_list_i)}')
#     total_list.extend(final_data_list_i)
# random.shuffle(total_list)
# total_list = total_list[::-1]
# print(f'total load num is : {len(total_list)}')

# input_s2s_data_path = "./data_config_s2s.yaml"
# output_s2s_data_path = "./data_config_s2s_ningxia.yaml"
# ningxia_root_dir = "/home/A02_tmpdata3/osum_s2s"
# data_dict = utils_file.load_dict_from_yaml(input_s2s_data_path)
# print(data_dict)
# for key, dict_i in data_dict.items():
#     if "path" not in dict_i:
#         print(f"path not in {key}")
#         continue
#     data_path_i = dict_i['path']
#     dict_i['huawei_path']= data_path_i
#     new_path = f'{ningxia_root_dir}/{key}/shards_list.txt'
#     dict_i['path'] = new_path
#     for key_i, value_i in dict_i.items():
#         value_i = value_i.strip() if isinstance(value_i, str) else value_i
#         dict_i[key_i] = value_i
# utils_file.write_dict_to_yaml(data_dict, output_s2s_data_path)

def handle_yaml(data_config_path):
    total_num = 0
    data_info_dict = utils_file.load_dict_from_yaml(data_config_path)
    for key, dict_i in data_info_dict.items():
        ningxia_path = dict_i.get("path", None)
        weight_num = float(dict_i.get("weight", 1))
        if ningxia_path is None:
            continue
        if utils_file.if_file_exist(ningxia_path):
            row_num = utils_file.do_get_file_rows_num_shell(ningxia_path)
            total_num += int(row_num * weight_num)
            if row_num == 0:
                utils_file.logging_warning(f"{key} : {ningxia_path} exist, but row num is 0")
            else:
                print(f"{key} : {ningxia_path} exist, row num is {row_num}, load num: {int(row_num * weight_num)}")
        else:
            utils_file.logging_error(f"{key} : {ningxia_path} not exist")
    utils_file.logging_info(f"{data_config_path} total num is {total_num}")

#check ningxia s2s数据情况
data_config_path = "./data_config_s2s_ningxia2.yaml"
handle_yaml(data_config_path)


data_config_path = "./data_config_s2t_ningxia2.yaml"
handle_yaml(data_config_path)



data_config_path = "./data_config_t2s_ningxia.yaml"
handle_yaml(data_config_path)


data_config_path = "./data_config_t2t_ningxia.yaml"
handle_yaml(data_config_path)


data_config_path = "./um.yaml"
handle_yaml(data_config_path)


